{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "notebookstart= time.time()\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "import os\n",
    "import gc\n",
    "# Models Packages\n",
    "from sklearn import metrics\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn import feature_selection\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn import preprocessing\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn import preprocessing\n",
    "from sklearn.metrics import roc_auc_score, roc_curve\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "import category_encoders as ce\n",
    "from imblearn.under_sampling import RandomUnderSampler\n",
    "from catboost import CatBoostClassifier\n",
    "# Gradient Boosting\n",
    "import lightgbm as lgb\n",
    "import xgboost as xgb\n",
    "import category_encoders as ce\n",
    "# Tf-Idf\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "from sklearn.pipeline import FeatureUnion\n",
    "from scipy.sparse import hstack, csr_matrix\n",
    "from nltk.corpus import stopwords \n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.ensemble import RandomForestRegressor \n",
    "# Viz\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np \n",
    "import pandas as pd \n",
    "from scipy.cluster.vq import kmeans2, whiten\n",
    "from sklearn.neighbors import NearestNeighbors, KNeighborsRegressor\n",
    "from catboost import CatBoostRegressor\n",
    "%matplotlib inline\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_rows = None\n",
    "EPS = 1e-100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv('/media/limbo/Home-Credit/data/application_train.csv.zip')\n",
    "test = pd.read_csv('/media/limbo/Home-Credit/data/application_test.csv')\n",
    "y = train['TARGET']\n",
    "\n",
    "n_train = train.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "d9 = pd.read_csv('../data/ds9.csv', header=0, index_col=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SK_ID_CURR</th>\n",
       "      <th>external_sources_mean</th>\n",
       "      <th>STCK_PREV_APP_ALL_</th>\n",
       "      <th>STCK_BER_ALL_</th>\n",
       "      <th>external_sources_nanmedian</th>\n",
       "      <th>external_sources_sum</th>\n",
       "      <th>external_sources_min</th>\n",
       "      <th>external_sources_max</th>\n",
       "      <th>AMT_CREDIT_INSTALLMENTS</th>\n",
       "      <th>NAME_EDUCATION_TYPE_CODE_GENDER_EXT_SOURCE_3_min_diff</th>\n",
       "      <th>...</th>\n",
       "      <th>x_5</th>\n",
       "      <th>x_6</th>\n",
       "      <th>x_7</th>\n",
       "      <th>x_8</th>\n",
       "      <th>x_9</th>\n",
       "      <th>x_10</th>\n",
       "      <th>retirement_age</th>\n",
       "      <th>long_employment</th>\n",
       "      <th>EXT_SOURCE_1</th>\n",
       "      <th>EXT_SOURCE_2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>100002</td>\n",
       "      <td>0.161787</td>\n",
       "      <td>0.133842</td>\n",
       "      <td>0.084131</td>\n",
       "      <td>0.139376</td>\n",
       "      <td>0.485361</td>\n",
       "      <td>0.083037</td>\n",
       "      <td>0.262949</td>\n",
       "      <td>16.461104</td>\n",
       "      <td>0.138849</td>\n",
       "      <td>...</td>\n",
       "      <td>48</td>\n",
       "      <td>35</td>\n",
       "      <td>140</td>\n",
       "      <td>70</td>\n",
       "      <td>450</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.083037</td>\n",
       "      <td>0.262949</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>100003</td>\n",
       "      <td>0.466757</td>\n",
       "      <td>0.040167</td>\n",
       "      <td>0.061941</td>\n",
       "      <td>0.466757</td>\n",
       "      <td>0.933513</td>\n",
       "      <td>0.311267</td>\n",
       "      <td>0.622246</td>\n",
       "      <td>36.234085</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>35</td>\n",
       "      <td>124</td>\n",
       "      <td>60</td>\n",
       "      <td>209</td>\n",
       "      <td>168</td>\n",
       "      <td>821</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.311267</td>\n",
       "      <td>0.622246</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>100004</td>\n",
       "      <td>0.642739</td>\n",
       "      <td>0.117715</td>\n",
       "      <td>0.068417</td>\n",
       "      <td>0.642739</td>\n",
       "      <td>1.285479</td>\n",
       "      <td>0.555912</td>\n",
       "      <td>0.729567</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>0.729039</td>\n",
       "      <td>...</td>\n",
       "      <td>60</td>\n",
       "      <td>112</td>\n",
       "      <td>107</td>\n",
       "      <td>128</td>\n",
       "      <td>73</td>\n",
       "      <td>114</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.555912</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>100006</td>\n",
       "      <td>0.650442</td>\n",
       "      <td>0.082768</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.650442</td>\n",
       "      <td>0.650442</td>\n",
       "      <td>0.650442</td>\n",
       "      <td>0.650442</td>\n",
       "      <td>10.532818</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>61</td>\n",
       "      <td>81</td>\n",
       "      <td>85</td>\n",
       "      <td>164</td>\n",
       "      <td>214</td>\n",
       "      <td>1016</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.650442</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>100007</td>\n",
       "      <td>0.322738</td>\n",
       "      <td>0.090020</td>\n",
       "      <td>0.065612</td>\n",
       "      <td>0.322738</td>\n",
       "      <td>0.322738</td>\n",
       "      <td>0.322738</td>\n",
       "      <td>0.322738</td>\n",
       "      <td>23.461618</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>59</td>\n",
       "      <td>15</td>\n",
       "      <td>21</td>\n",
       "      <td>193</td>\n",
       "      <td>32</td>\n",
       "      <td>612</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.322738</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 2044 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   SK_ID_CURR  external_sources_mean  STCK_PREV_APP_ALL_  STCK_BER_ALL_  \\\n",
       "0      100002               0.161787            0.133842       0.084131   \n",
       "1      100003               0.466757            0.040167       0.061941   \n",
       "2      100004               0.642739            0.117715       0.068417   \n",
       "3      100006               0.650442            0.082768            NaN   \n",
       "4      100007               0.322738            0.090020       0.065612   \n",
       "\n",
       "   external_sources_nanmedian  external_sources_sum  external_sources_min  \\\n",
       "0                    0.139376              0.485361              0.083037   \n",
       "1                    0.466757              0.933513              0.311267   \n",
       "2                    0.642739              1.285479              0.555912   \n",
       "3                    0.650442              0.650442              0.650442   \n",
       "4                    0.322738              0.322738              0.322738   \n",
       "\n",
       "   external_sources_max  AMT_CREDIT_INSTALLMENTS  \\\n",
       "0              0.262949                16.461104   \n",
       "1              0.622246                36.234085   \n",
       "2              0.729567                20.000000   \n",
       "3              0.650442                10.532818   \n",
       "4              0.322738                23.461618   \n",
       "\n",
       "   NAME_EDUCATION_TYPE_CODE_GENDER_EXT_SOURCE_3_min_diff      ...       x_5  \\\n",
       "0                                           0.138849          ...        48   \n",
       "1                                                NaN          ...        35   \n",
       "2                                           0.729039          ...        60   \n",
       "3                                                NaN          ...        61   \n",
       "4                                                NaN          ...        59   \n",
       "\n",
       "   x_6  x_7  x_8  x_9  x_10  retirement_age  long_employment  EXT_SOURCE_1  \\\n",
       "0   35  140   70  450    13               1                1      0.083037   \n",
       "1  124   60  209  168   821               0                1      0.311267   \n",
       "2  112  107  128   73   114               0                1           NaN   \n",
       "3   81   85  164  214  1016               0                0           NaN   \n",
       "4   15   21  193   32   612               0                0           NaN   \n",
       "\n",
       "   EXT_SOURCE_2  \n",
       "0      0.262949  \n",
       "1      0.622246  \n",
       "2      0.555912  \n",
       "3      0.650442  \n",
       "4      0.322738  \n",
       "\n",
       "[5 rows x 2044 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d9.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "no_hc = pd.read_csv('../data/no_hc_loan_cust.csv', index_col=None, header=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "ind = d9['SK_ID_CURR'].isin(no_hc.values[:, 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "external_sources_mean\n",
      "external_sources_nanmedian\n",
      "external_sources_sum\n",
      "external_sources_min\n",
      "external_sources_max\n",
      "NAME_EDUCATION_TYPE_CODE_GENDER_EXT_SOURCE_3_min_diff\n",
      "NAME_FAMILY_STATUS_NAME_EDUCATION_TYPE_EXT_SOURCE_3_mean_diff\n",
      "NAME_FAMILY_STATUS_NAME_EDUCATION_TYPE_EXT_SOURCE_3_max_abs_diff\n",
      "NAME_FAMILY_STATUS_CODE_GENDER_EXT_SOURCE_3_mean_diff\n",
      "NAME_FAMILY_STATUS_CODE_GENDER_EXT_SOURCE_2_min_diff\n",
      "NAME_EDUCATION_TYPE_CODE_GENDER_EXT_SOURCE_2_max_abs_diff\n",
      "NAME_EDUCATION_TYPE_CODE_GENDER_EXT_SOURCE_3_max_abs_diff\n",
      "EXT_SOURCE_3.x\n",
      "NAME_FAMILY_STATUS_CODE_GENDER_EXT_SOURCE_2_min_abs_diff\n",
      "NAME_EDUCATION_TYPE_CODE_GENDER_EXT_SOURCE_2_max_diff\n",
      "NAME_FAMILY_STATUS_CODE_GENDER_EXT_SOURCE_2_max_diff\n",
      "NAME_EDUCATION_TYPE_CODE_GENDER_EXT_SOURCE_2_min_diff\n",
      "NAME_FAMILY_STATUS_CODE_GENDER_EXT_SOURCE_3_max_abs_diff\n",
      "NAME_FAMILY_STATUS_NAME_EDUCATION_TYPE_EXT_SOURCE_3_min_diff\n",
      "NAME_FAMILY_STATUS_NAME_EDUCATION_TYPE_EXT_SOURCE_3_min_abs_diff\n",
      "EXT_1_DIV_BIRTH_SURESH.x\n",
      "NAME_FAMILY_STATUS_NAME_EDUCATION_TYPE_EXT_SOURCE_2_min_abs_diff\n",
      "NAME_FAMILY_STATUS_NAME_EDUCATION_TYPE_EXT_SOURCE_3_max_diff\n",
      "EXT_MEAN\n",
      "NAME_EDUCATION_TYPE_CODE_GENDER_EXT_SOURCE_3_mean_diff\n",
      "NAME_FAMILY_STATUS_CODE_GENDER_EXT_SOURCE_2_max_abs_diff\n",
      "NAME_FAMILY_STATUS_CODE_GENDER_EXT_SOURCE_3_min_diff\n",
      "external_sources_weighted\n",
      "NAME_FAMILY_STATUS_NAME_EDUCATION_TYPE_EXT_SOURCE_2_max_diff\n",
      "EXT_SOURCE_2.x\n",
      "NAME_FAMILY_STATUS_NAME_EDUCATION_TYPE_EXT_SOURCE_2_min_diff\n",
      "NAME_EDUCATION_TYPE_CODE_GENDER_EXT_SOURCE_3_max_diff\n",
      "NAME_FAMILY_STATUS_NAME_EDUCATION_TYPE_EXT_SOURCE_3_mean_abs_diff\n",
      "NAME_FAMILY_STATUS_CODE_GENDER_EXT_SOURCE_3_max_diff\n",
      "NAME_FAMILY_STATUS_CODE_GENDER_EXT_SOURCE_3_mean_abs_diff\n",
      "NAME_FAMILY_STATUS_NAME_EDUCATION_TYPE_EXT_SOURCE_2_max_abs_diff\n",
      "NAME_EDUCATION_TYPE_CODE_GENDER_EXT_SOURCE_2_mean_diff\n",
      "NAME_EDUCATION_TYPE_OCCUPATION_TYPE_EXT_SOURCE_2_mean_diff\n",
      "CODE_GENDER_NAME_EDUCATION_TYPE_OCCUPATION_TYPE_REG_CITY_NOT_WORK_CITY_EXT_SOURCE_2_mean_diff\n",
      "OCCUPATION_TYPE_EXT_SOURCE_2_mean_abs_diff\n",
      "NAME_FAMILY_STATUS_CODE_GENDER_EXT_SOURCE_2_mean_abs_diff\n",
      "NAME_FAMILY_STATUS_CODE_GENDER_EXT_SOURCE_2_mean_diff\n",
      "OCCUPATION_TYPE_EXT_SOURCE_2_mean_diff\n",
      "EXT_3_DIV_BIRTH_SURESH.x\n",
      "NAME_FAMILY_STATUS_CODE_GENDER_EXT_SOURCE_1_max_abs_diff\n",
      "NAME_EDUCATION_TYPE_OCCUPATION_TYPE_EXT_SOURCE_2_mean_abs_diff\n",
      "CODE_GENDER_NAME_EDUCATION_TYPE_OCCUPATION_TYPE_REG_CITY_NOT_WORK_CITY_EXT_SOURCE_2_mean_abs_diff\n",
      "NAME_FAMILY_STATUS_NAME_EDUCATION_TYPE_EXT_SOURCE_2_mean_diff\n",
      "NAME_FAMILY_STATUS_NAME_EDUCATION_TYPE_EXT_SOURCE_1_max_diff\n",
      "NAME_EDUCATION_TYPE_CODE_GENDER_EXT_SOURCE_1_max_abs_diff\n",
      "NAME_EDUCATION_TYPE_CODE_GENDER_EXT_SOURCE_1_min_diff\n",
      "NAME_EDUCATION_TYPE_CODE_GENDER_EXT_SOURCE_3_mean_abs_diff\n",
      "NAME_EDUCATION_TYPE_CODE_GENDER_EXT_SOURCE_1_min_abs_diff\n",
      "NAME_FAMILY_STATUS_NAME_EDUCATION_TYPE_EXT_SOURCE_1_min_abs_diff\n",
      "NAME_EDUCATION_TYPE_CODE_GENDER_EXT_SOURCE_2_mean_abs_diff\n",
      "NAME_FAMILY_STATUS_CODE_GENDER_EXT_SOURCE_1_mean_diff\n",
      "NAME_FAMILY_STATUS_NAME_EDUCATION_TYPE_EXT_SOURCE_2_mean_abs_diff\n",
      "NAME_FAMILY_STATUS_CODE_GENDER_EXT_SOURCE_1_max_diff\n",
      "NAME_FAMILY_STATUS_CODE_GENDER_EXT_SOURCE_1_min_diff\n",
      "CODE_GENDER_ORGANIZATION_TYPE_EXT_SOURCE_1_mean_diff\n",
      "NAME_FAMILY_STATUS_NAME_EDUCATION_TYPE_EXT_SOURCE_1_max_abs_diff\n",
      "NAME_EDUCATION_TYPE_CODE_GENDER_EXT_SOURCE_1_max_diff\n",
      "NAME_FAMILY_STATUS_CODE_GENDER_EXT_SOURCE_1_min_abs_diff\n",
      "NAME_FAMILY_STATUS_NAME_EDUCATION_TYPE_EXT_SOURCE_1_min_diff\n",
      "OCCUPATION_TYPE_EXT_SOURCE_3_mean_diff\n",
      "NAME_EDUCATION_TYPE_OCCUPATION_TYPE_EXT_SOURCE_3_mean_diff\n",
      "EXT_2_DIV_BIRTH_SURESH.x\n",
      "OCCUPATION_TYPE_EXT_SOURCE_3_mean_abs_diff\n",
      "NAME_EDUCATION_TYPE_OCCUPATION_TYPE_EXT_SOURCE_3_mean_abs_diff\n",
      "NAME_EDUCATION_TYPE_CODE_GENDER_EXT_SOURCE_1_mean_diff\n",
      "EXT_SOURCE_1.x\n",
      "NAME_FAMILY_STATUS_NAME_EDUCATION_TYPE_EXT_SOURCE_1_mean_diff\n",
      "NAME_EDUCATION_TYPE_CODE_GENDER_EXT_SOURCE_1_mean_abs_diff\n",
      "NAME_FAMILY_STATUS_NAME_EDUCATION_TYPE_EXT_SOURCE_1_mean_abs_diff\n",
      "CODE_GENDER_ORGANIZATION_TYPE_EXT_SOURCE_1_mean_abs_diff\n",
      "YEARS_BEGINEXPLUATATION_MEDI.x\n",
      "CODE_GENDER_NAME_EDUCATION_TYPE_OCCUPATION_TYPE_REG_CITY_NOT_WORK_CITY_EXT_SOURCE_1_mean_diff\n",
      "EXT_SOURCE_1.y\n",
      "EXT_SOURCE_2.y\n",
      "EXT_SOURCE_3.y\n",
      "YEARS_BEGINEXPLUATATION_AVG\n",
      "YEARS_BEGINEXPLUATATION_MEDI.y\n",
      "YEARS_BEGINEXPLUATATION_MODE\n",
      "app EXT_SOURCE mean\n",
      "app EXT_SOURCE std\n",
      "app EXT_SOURCE prod\n",
      "app EXT_SOURCE_1 * EXT_SOURCE_2\n",
      "app EXT_SOURCE_1 * EXT_SOURCE_3\n",
      "app EXT_SOURCE_2 * EXT_SOURCE_3\n",
      "app EXT_SOURCE_1 * DAYS_EMPLOYED\n",
      "app EXT_SOURCE_2 * DAYS_EMPLOYED\n",
      "app EXT_SOURCE_3 * DAYS_EMPLOYED\n",
      "app EXT_SOURCE_1 / DAYS_BIRTH\n",
      "app EXT_SOURCE_2 / DAYS_BIRTH\n",
      "app EXT_SOURCE_3 / DAYS_BIRTH\n",
      "TOTAL_CC_EXPENSES_SURESH\n",
      "CUSTOMER_EXT1_OVERREGION_SURESH\n",
      "CUSTOMER_EXT2_OVERREGION_SURESH\n",
      "CUSTOMER_EXT3_OVERREGION_SURESH\n",
      "EXTERNAL_SOURCE_1_MEDIAN_SURESH\n",
      "EXTERNAL_SOURCE_2_MEDIAN_SURESH\n",
      "EXTERNAL_SOURCE_3_MEDIAN_SURESH\n",
      "EXT_3_TO_REGION_MEAN\n",
      "EXT_2_TO_REGION_MEAN\n",
      "EXT_1_TO_REGION_MEAN\n",
      "NAME_CONTRACT_TYPE-EXT_SOURCE_1_mean\n",
      "EXT_SOURCE_1\n",
      "EXT_SOURCE_2\n"
     ]
    }
   ],
   "source": [
    "for f in d9.columns:\n",
    "    if 'ex' in f:\n",
    "        print(f)\n",
    "        d9.loc[ind.values, f] = np.nan\n",
    "    if 'EX' in f:\n",
    "        print(f)\n",
    "        d9.loc[ind.values, f] = np.nan\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SK_ID_CURR</th>\n",
       "      <th>external_sources_mean</th>\n",
       "      <th>STCK_PREV_APP_ALL_</th>\n",
       "      <th>STCK_BER_ALL_</th>\n",
       "      <th>external_sources_nanmedian</th>\n",
       "      <th>external_sources_sum</th>\n",
       "      <th>external_sources_min</th>\n",
       "      <th>external_sources_max</th>\n",
       "      <th>AMT_CREDIT_INSTALLMENTS</th>\n",
       "      <th>NAME_EDUCATION_TYPE_CODE_GENDER_EXT_SOURCE_3_min_diff</th>\n",
       "      <th>...</th>\n",
       "      <th>x_5</th>\n",
       "      <th>x_6</th>\n",
       "      <th>x_7</th>\n",
       "      <th>x_8</th>\n",
       "      <th>x_9</th>\n",
       "      <th>x_10</th>\n",
       "      <th>retirement_age</th>\n",
       "      <th>long_employment</th>\n",
       "      <th>EXT_SOURCE_1</th>\n",
       "      <th>EXT_SOURCE_2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>100002</td>\n",
       "      <td>0.161787</td>\n",
       "      <td>0.133842</td>\n",
       "      <td>0.084131</td>\n",
       "      <td>0.139376</td>\n",
       "      <td>0.485361</td>\n",
       "      <td>0.083037</td>\n",
       "      <td>0.262949</td>\n",
       "      <td>16.461104</td>\n",
       "      <td>0.138849</td>\n",
       "      <td>...</td>\n",
       "      <td>48</td>\n",
       "      <td>35</td>\n",
       "      <td>140</td>\n",
       "      <td>70</td>\n",
       "      <td>450</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.083037</td>\n",
       "      <td>0.262949</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>100003</td>\n",
       "      <td>0.466757</td>\n",
       "      <td>0.040167</td>\n",
       "      <td>0.061941</td>\n",
       "      <td>0.466757</td>\n",
       "      <td>0.933513</td>\n",
       "      <td>0.311267</td>\n",
       "      <td>0.622246</td>\n",
       "      <td>36.234085</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>35</td>\n",
       "      <td>124</td>\n",
       "      <td>60</td>\n",
       "      <td>209</td>\n",
       "      <td>168</td>\n",
       "      <td>821</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.311267</td>\n",
       "      <td>0.622246</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>100004</td>\n",
       "      <td>0.642739</td>\n",
       "      <td>0.117715</td>\n",
       "      <td>0.068417</td>\n",
       "      <td>0.642739</td>\n",
       "      <td>1.285479</td>\n",
       "      <td>0.555912</td>\n",
       "      <td>0.729567</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>0.729039</td>\n",
       "      <td>...</td>\n",
       "      <td>60</td>\n",
       "      <td>112</td>\n",
       "      <td>107</td>\n",
       "      <td>128</td>\n",
       "      <td>73</td>\n",
       "      <td>114</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.555912</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>100006</td>\n",
       "      <td>0.650442</td>\n",
       "      <td>0.082768</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.650442</td>\n",
       "      <td>0.650442</td>\n",
       "      <td>0.650442</td>\n",
       "      <td>0.650442</td>\n",
       "      <td>10.532818</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>61</td>\n",
       "      <td>81</td>\n",
       "      <td>85</td>\n",
       "      <td>164</td>\n",
       "      <td>214</td>\n",
       "      <td>1016</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.650442</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>100007</td>\n",
       "      <td>0.322738</td>\n",
       "      <td>0.090020</td>\n",
       "      <td>0.065612</td>\n",
       "      <td>0.322738</td>\n",
       "      <td>0.322738</td>\n",
       "      <td>0.322738</td>\n",
       "      <td>0.322738</td>\n",
       "      <td>23.461618</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>59</td>\n",
       "      <td>15</td>\n",
       "      <td>21</td>\n",
       "      <td>193</td>\n",
       "      <td>32</td>\n",
       "      <td>612</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.322738</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 2044 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   SK_ID_CURR  external_sources_mean  STCK_PREV_APP_ALL_  STCK_BER_ALL_  \\\n",
       "0      100002               0.161787            0.133842       0.084131   \n",
       "1      100003               0.466757            0.040167       0.061941   \n",
       "2      100004               0.642739            0.117715       0.068417   \n",
       "3      100006               0.650442            0.082768            NaN   \n",
       "4      100007               0.322738            0.090020       0.065612   \n",
       "\n",
       "   external_sources_nanmedian  external_sources_sum  external_sources_min  \\\n",
       "0                    0.139376              0.485361              0.083037   \n",
       "1                    0.466757              0.933513              0.311267   \n",
       "2                    0.642739              1.285479              0.555912   \n",
       "3                    0.650442              0.650442              0.650442   \n",
       "4                    0.322738              0.322738              0.322738   \n",
       "\n",
       "   external_sources_max  AMT_CREDIT_INSTALLMENTS  \\\n",
       "0              0.262949                16.461104   \n",
       "1              0.622246                36.234085   \n",
       "2              0.729567                20.000000   \n",
       "3              0.650442                10.532818   \n",
       "4              0.322738                23.461618   \n",
       "\n",
       "   NAME_EDUCATION_TYPE_CODE_GENDER_EXT_SOURCE_3_min_diff      ...       x_5  \\\n",
       "0                                           0.138849          ...        48   \n",
       "1                                                NaN          ...        35   \n",
       "2                                           0.729039          ...        60   \n",
       "3                                                NaN          ...        61   \n",
       "4                                                NaN          ...        59   \n",
       "\n",
       "   x_6  x_7  x_8  x_9  x_10  retirement_age  long_employment  EXT_SOURCE_1  \\\n",
       "0   35  140   70  450    13               1                1      0.083037   \n",
       "1  124   60  209  168   821               0                1      0.311267   \n",
       "2  112  107  128   73   114               0                1           NaN   \n",
       "3   81   85  164  214  1016               0                0           NaN   \n",
       "4   15   21  193   32   612               0                0           NaN   \n",
       "\n",
       "   EXT_SOURCE_2  \n",
       "0      0.262949  \n",
       "1      0.622246  \n",
       "2      0.555912  \n",
       "3      0.650442  \n",
       "4      0.322738  \n",
       "\n",
       "[5 rows x 2044 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d9.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "del d9['TARGET.x']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_file_path = \"Level_1_stack/test_lgbm_big_2.csv\"\n",
    "validation_file_path = 'Level_1_stack/validation_lgbm_big_2.csv'\n",
    "num_folds = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_features = train_features.replace([np.inf, -np.inf], np.nan).fillna(-999, inplace=False)\n",
    "# test_features = test_features.replace([np.inf, -np.inf], np.nan).fillna(-999, inplace=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting LightGBM. Train shape: (307511, 2043), test shape: (48744, 2043)\n",
      "(307511, 2043) (307511, 2042)\n",
      "Fold  1 AUC : 0.781552\n",
      "Fold  2 AUC : 0.717248\n",
      "Fold  3 AUC : 0.773391\n",
      "Fold  4 AUC : 0.777863\n",
      "Fold  5 AUC : 0.781452\n"
     ]
    }
   ],
   "source": [
    "gc.collect()\n",
    "train_df = d9.iloc[0:n_train]\n",
    "test_df = d9.iloc[n_train:]\n",
    "\n",
    "val_df = train[['SK_ID_CURR', 'TARGET']].copy()\n",
    "\n",
    "print(\"Starting LightGBM. Train shape: {}, test shape: {}\".format(train_df.shape, test_df.shape))\n",
    "gc.collect()\n",
    "# Cross validation model\n",
    "folds = KFold(n_splits=num_folds, shuffle=True, random_state=1001)\n",
    "# Create arrays and dataframes to store results\n",
    "oof_preds = np.zeros(train_df.shape[0])\n",
    "sub_preds = np.zeros(test_df.shape[0])\n",
    "feature_importance_df = pd.DataFrame()\n",
    "\n",
    "feats = [f for f in train_df.columns if f not in ['TARGET','SK_ID_CURR','SK_ID_BUREAU','SK_ID_PREV','index']]\n",
    "\n",
    "x_train = train_df[feats].values\n",
    "x_test = test_df[feats].values\n",
    "\n",
    "print(train_df.shape, x_train.shape)\n",
    "\n",
    "\n",
    "for n_fold, (train_idx, valid_idx) in enumerate(folds.split(x_train, train['TARGET'])):\n",
    "    dtrain = lgb.Dataset(data=x_train[train_idx], \n",
    "                         label=train['TARGET'].iloc[train_idx], \n",
    "                         free_raw_data=False, silent=True)\n",
    "    dvalid = lgb.Dataset(data=x_train[valid_idx], \n",
    "                         label=train['TARGET'].iloc[valid_idx], \n",
    "                         free_raw_data=False, silent=True)\n",
    "\n",
    "    params =  {\n",
    "        'task': 'train',\n",
    "        'boosting_type': 'goss',\n",
    "        'objective': 'regression',\n",
    "        'metric': 'mape',\n",
    "        'max_depth': -1,\n",
    "        'nthreads': 4,\n",
    "        'subsample': 0.9,\n",
    "        \"min_child_weight\": 2 ** 6,\n",
    "        'num_leaves': 2  ** 5,\n",
    "        'feature_fraction': 0.25,\n",
    "        'xgboost_dart_mode': True,\n",
    "        'seed': 333,\n",
    "#         'bagging_fraction': 0.5,\n",
    "        # 'bagging_freq': 5,\n",
    "        \"reg_lambda\": 100,\n",
    "        'learning_rate': 0.02,\n",
    "        'verbose': -1\n",
    "    } \n",
    "    \n",
    "   \n",
    "    clf = lgb.train(\n",
    "        params=params,\n",
    "        train_set=dtrain,\n",
    "        num_boost_round=10000,\n",
    "        valid_sets=[dtrain, dvalid],\n",
    "        early_stopping_rounds=100,\n",
    "        verbose_eval=False\n",
    "    )\n",
    "\n",
    "    oof_preds[valid_idx] = clf.predict(dvalid.data)\n",
    "    sub_preds += clf.predict(x_test) / folds.n_splits\n",
    "\n",
    "    print('Fold %2d AUC : %.6f' % (n_fold + 1, roc_auc_score(dvalid.label, oof_preds[valid_idx])))\n",
    "    del clf, dtrain, dvalid\n",
    "    gc.collect()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Full AUC score 0.743388\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print('Full AUC score %.6f' % roc_auc_score(train['TARGET'], oof_preds))\n",
    "# Write submission file and plot feature importance\n",
    "\n",
    "sub_df = test[['SK_ID_CURR']].copy()\n",
    "sub_df['TARGET'] = sub_preds\n",
    "sub_df[['SK_ID_CURR', 'TARGET']].to_csv(test_file_path, index= False)\n",
    "\n",
    "\n",
    "val_df['TARGET'] = oof_preds\n",
    "val_df[['SK_ID_CURR', 'TARGET']].to_csv(validation_file_path, index= False)\n",
    "\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
